import time
import learning as l
import os
import utils

from scipy import sparse
from Categories import Categories
from Words import Words
from Files import Files
from CatWordsMatrix import CatWordsMatrix
from clusters import Clusters
from stemmer import PorterStemmer
from nbm import NBM

"Ensemble des stopwords"
stopwords = set([])

"Nombre des fichiers lus"
folders = ('awards_1990', 'awards_1991')#, 'awards_1992')

"Instances de Words"
allwords = Words()

"Instance de Categories"
categories = Categories()

"Instance de Files"
files = Files()

"Matrice"
featuresMatrix = sparse.lil_matrix((1, 1))

"Instance de condensed matrix"
matrix2 = CatWordsMatrix(utils.DATA_PATH)

if os.access(utils.DATA_PATH, os.F_OK):

    utils.printTime("Load data structures from files START")
    stopwords = utils.load('stopwords.ser')
    allwords = utils.load('allwords.ser')
    folders = utils.load('folders.ser')
    categories = utils.load('categories.ser')
    files = utils.load('files.ser')
    featuresMatrix = utils.load('featuresMatrix.ser')
    matrix2 = utils.load('matrix2.ser')
    cls = utils.load('cls.ser')
    reFeaturesMatrix = utils.load('reFeaturesMatrix.ser')
    utils.printTime("Load data structures from files STOP")
else:
    os.makedirs(utils.DATA_PATH)

    utils.printTime("START")

    utils.printTime("Read StopWords START")
    stopwords = l.readStopWords()
    utils.printTime("Read StopWords STOP")

    utils.printTime("Read Learning Files START")
    l.readFiles(l.PATH, folders, allwords, categories, stopwords, files)    
    utils.printTime("Read Learning Files STOP")

    utils.printTime("Load Matrix START")
    featuresMatrix = l.loadMatrix(l.PATH, allwords, categories, files).tocsr()
    utils.printTime("Load Matrix STOP")

    utils.printTime("Load Matrix2 START")
    matrix2.loadMatrix(categories, allwords, files)
    utils.printTime("Load Matrix2 STOP")

    utils.printTime("Prune matrixFreq START")
    matrix2.pruneMatrix(l.MIN_VAR)
    utils.printTime("Prune matrixFreq STOP")

    utils.printTime("Clustering START")
    cls = Clusters(matrix2.matrixFreq.toarray())
    utils.printTime("Clustering STOP")

    utils.printTime("Rebuild featuresMatrix START")
    reFeaturesMatrix = l.rebuildFeaturesMatrix(featuresMatrix, matrix2, cls)
    utils.printTime("Rebuild featuresMatrix STOP")
    
    utils.printTime("Save data structures START")
    utils.save('stopwords.ser', stopwords)    
    utils.save('allwords.ser', allwords)
    utils.save('folders.ser', folders)
    utils.save('categories.ser', categories)
    utils.save('files.ser', files)
    utils.save('featuresMatrix.ser', featuresMatrix)
    utils.save('matrix2.ser', matrix2)    
    utils.save('cls.ser', cls)    
    utils.save('reFeaturesMatrix.ser', reFeaturesMatrix)    
    utils.printTime("Save data structures STOP")


#nbmClassifier = NBM(reFeaturesMatrix, cls, allwords, matrix2)

#p = PorterStemmer()
#hits = 0
#count = 0
#for i in os.listdir('../../Part1/awards_1994/awd_1994_00'):
#    fdata = l.readFileData('../../Part1/awards_1994/awd_1994_00', i)
#    c = l.readCategory(fdata)
#    ws = l.readWords(stopwords, fdata, p)
#    candidate = nbmClassifier.classify(ws)
#    count += 1
#    if c in categories.bag.keys(): 
#        catId = categories.bag[c][0]
#        if catId in cls.positions[candidate]:
#            hits += 1
#print 'hits : ', hits
#print 'succes : ', float(hits * 100) / os.listdir('../../Part1/awards_1994/awd_1994_00').__len__(), '%'



#"On efface la matrice"
#del matrix
#print "[", time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "] STOP"
